{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parallel computing on multivectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Distributed\n",
    "using DistributedArrays\n",
    "addprocs(4);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "@everywhere using JOLI\n",
    "@everywhere using LinearAlgebra"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setup problem size, define operator, and input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "M=8;\n",
    "NVC=15;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type: joLinearFunction\n",
      "Name: joDFT\n",
      "Size: (8, 8)\n",
      " DDT: Float64\n",
      " RDT: Complex{Float64}\n"
     ]
    }
   ],
   "source": [
    "A=joDFT(M,planned=false);\n",
    "show(A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8×15 Array{Float64,2}:\n",
       " -0.270754  -1.3369     -1.06987    …  -0.558707  -1.60898    -0.0924695\n",
       " -0.60623   -1.67037     0.0420038      0.342735   0.277882   -0.250557 \n",
       "  0.98971   -0.0660405  -2.57101       -0.176547  -0.427845    0.549751 \n",
       "  0.372959   1.26178     0.153234      -1.1445    -1.51067     0.0844326\n",
       " -1.19674   -0.136317   -0.571931       0.110122   0.111624    0.662711 \n",
       " -1.01415   -2.42453    -1.363      …   2.26607    0.153304    0.0213506\n",
       "  1.53771    0.673964    0.95488        0.823318  -0.127601    0.773563 \n",
       " -0.251367   0.715949   -0.0310003     -1.55734   -0.0625879   1.02815  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x=randn(M,NVC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Either Ararys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Query/Set parallel mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parallel mode is set to :SA (shared arrays)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       ":SA"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pmode=jo_PAmode(verbose=true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       ":SA"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jo_PAmode(:SA) # :SA for shared or :DA for distributed\n",
    "pmode=jo_PAmode()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define parallel setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "joPAsetup: joPAsetup\n",
      " DataType: Float64\n",
      " Dims    : (8, 15)\n",
      " Chunks  : [1, 4]\n",
      " Workers : [2, 3, 4, 5]\n",
      "  Worker/ranges:   2 (1:8, 1:4)\n",
      "  Worker/ranges:   3 (1:8, 5:8)\n",
      "  Worker/ranges:   4 (1:8, 9:12)\n",
      "  Worker/ranges:   5 (1:8, 13:15)\n"
     ]
    }
   ],
   "source": [
    "ps=joPAsetup((M,NVC));\n",
    "show(ps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define parallel operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type: joSAdistributedLinearOperator\n",
      "Name: joSAdistributedLinearOperator(joDFT)\n",
      "Size: (8, 8)\n",
      " NVC: 15\n",
      " DDT: Float64\n",
      " RDT: Complex{Float64}\n"
     ]
    }
   ],
   "source": [
    "pA=joPAdistributedLinOp(A,ps);\n",
    "show(pA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define shared/distributed array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8×15 SharedArrays.SharedArray{Float64,2}:\n",
       " -0.270754  -1.3369     -1.06987    …  -0.558707  -1.60898    -0.0924695\n",
       " -0.60623   -1.67037     0.0420038      0.342735   0.277882   -0.250557 \n",
       "  0.98971   -0.0660405  -2.57101       -0.176547  -0.427845    0.549751 \n",
       "  0.372959   1.26178     0.153234      -1.1445    -1.51067     0.0844326\n",
       " -1.19674   -0.136317   -0.571931       0.110122   0.111624    0.662711 \n",
       " -1.01415   -2.42453    -1.363      …   2.26607    0.153304    0.0213506\n",
       "  1.53771    0.673964    0.95488        0.823318  -0.127601    0.773563 \n",
       " -0.251367   0.715949   -0.0310003     -1.55734   -0.0625879   1.02815  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@everywhere function fillin(x,n)\n",
    "    println(myid(),n,(length(n[1]),length(n[2])))\n",
    "    x[n...]\n",
    "end\n",
    "\n",
    "if pmode==:SA\n",
    "    px=salloc(ps)\n",
    "    px[:,:]=x[:,:]\n",
    "elseif pmode==:DA\n",
    "    px=dfill(r->fillin(x,r),ps)\n",
    "end\n",
    "px"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Just parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8×15 SharedArrays.SharedArray{Complex{Float64},2}:\n",
       " -0.155163+0.0im         -1.05446+0.0im        …   0.981796+0.0im     \n",
       "  0.273285-0.0643151im  -0.372388-0.0633667im     -0.099043+0.383037im\n",
       "  -1.41242+0.615882im   -0.735795+2.147im         -0.266251+0.474396im\n",
       "  0.381486-0.45181im    -0.476554-0.586629im       -0.43495+0.224778im\n",
       "  0.904644+0.0im         0.442604+0.0im            0.357151+0.0im     \n",
       "  0.381486+0.45181im    -0.476554+0.586629im   …   -0.43495-0.224778im\n",
       "  -1.41242-0.615882im   -0.735795-2.147im         -0.266251-0.474396im\n",
       "  0.273285+0.0643151im  -0.372388+0.0633667im     -0.099043-0.383037im"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "py=pA*px"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Serial to serial via parallel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define distrtibuting and gathering operators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type: joSAdistribute\n",
      "Name: joSAdistributeMV:15\n",
      "Size: (8, 8)\n",
      " NVC: 15\n",
      " DDT: Float64\n",
      " RDT: Float64\n"
     ]
    }
   ],
   "source": [
    "pD=joPAdistribute(pA);\n",
    "show(pD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type: joSAgather\n",
      "Name: joSAgatherMV:15\n",
      "Size: (8, 8)\n",
      " NVC: 15\n",
      " DDT: Complex{Float64}\n",
      " RDT: Complex{Float64}\n"
     ]
    }
   ],
   "source": [
    "pG=joPAgather(pA);\n",
    "show(pG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Distribute data, apply operator, gather data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8×15 SharedArrays.SharedArray{Float64,2}:\n",
       " -0.270754  -1.3369     -1.06987    …  -0.558707  -1.60898    -0.0924695\n",
       " -0.60623   -1.67037     0.0420038      0.342735   0.277882   -0.250557 \n",
       "  0.98971   -0.0660405  -2.57101       -0.176547  -0.427845    0.549751 \n",
       "  0.372959   1.26178     0.153234      -1.1445    -1.51067     0.0844326\n",
       " -1.19674   -0.136317   -0.571931       0.110122   0.111624    0.662711 \n",
       " -1.01415   -2.42453    -1.363      …   2.26607    0.153304    0.0213506\n",
       "  1.53771    0.673964    0.95488        0.823318  -0.127601    0.773563 \n",
       " -0.251367   0.715949   -0.0310003     -1.55734   -0.0625879   1.02815  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "px=pD*x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8×15 SharedArrays.SharedArray{Complex{Float64},2}:\n",
       " -0.155163+0.0im         -1.05446+0.0im        …   0.981796+0.0im     \n",
       "  0.273285-0.0643151im  -0.372388-0.0633667im     -0.099043+0.383037im\n",
       "  -1.41242+0.615882im   -0.735795+2.147im         -0.266251+0.474396im\n",
       "  0.381486-0.45181im    -0.476554-0.586629im       -0.43495+0.224778im\n",
       "  0.904644+0.0im         0.442604+0.0im            0.357151+0.0im     \n",
       "  0.381486+0.45181im    -0.476554+0.586629im   …   -0.43495-0.224778im\n",
       "  -1.41242-0.615882im   -0.735795-2.147im         -0.266251-0.474396im\n",
       "  0.273285+0.0643151im  -0.372388+0.0633667im     -0.099043-0.383037im"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "py=pA*px"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8×15 Array{Complex{Float64},2}:\n",
       " -0.155163+0.0im         -1.05446+0.0im        …   0.981796+0.0im     \n",
       "  0.273285-0.0643151im  -0.372388-0.0633667im     -0.099043+0.383037im\n",
       "  -1.41242+0.615882im   -0.735795+2.147im         -0.266251+0.474396im\n",
       "  0.381486-0.45181im    -0.476554-0.586629im       -0.43495+0.224778im\n",
       "  0.904644+0.0im         0.442604+0.0im            0.357151+0.0im     \n",
       "  0.381486+0.45181im    -0.476554+0.586629im   …   -0.43495-0.224778im\n",
       "  -1.41242-0.615882im   -0.735795-2.147im         -0.266251-0.474396im\n",
       "  0.273285+0.0643151im  -0.372388+0.0633667im     -0.099043-0.383037im"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y=pG*py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or all together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type: joLinearOperator\n",
      "Name: ((joSAgatherMV:15*joSAdistributedLinearOperator(joDFT))*joSAdistributeMV:15)\n",
      "Size: (8, 8)\n",
      " DDT: Float64\n",
      " RDT: Complex{Float64}\n"
     ]
    }
   ],
   "source": [
    "ps2s=pG*pA*pD\n",
    "show(ps2s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8×15 Array{Complex{Float64},2}:\n",
       " -0.155163+0.0im         -1.05446+0.0im        …   0.981796+0.0im     \n",
       "  0.273285-0.0643151im  -0.372388-0.0633667im     -0.099043+0.383037im\n",
       "  -1.41242+0.615882im   -0.735795+2.147im         -0.266251+0.474396im\n",
       "  0.381486-0.45181im    -0.476554-0.586629im       -0.43495+0.224778im\n",
       "  0.904644+0.0im         0.442604+0.0im            0.357151+0.0im     \n",
       "  0.381486+0.45181im    -0.476554+0.586629im   …   -0.43495-0.224778im\n",
       "  -1.41242-0.615882im   -0.735795-2.147im         -0.266251-0.474396im\n",
       "  0.273285+0.0643151im  -0.372388+0.0633667im     -0.099043-0.383037im"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y=ps2s*x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.3.1",
   "language": "julia",
   "name": "julia-1.3"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.3.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
